# -*- coding: utf-8 -*-
"""
Created on 20180725 22:02
@author: danson
tips: change from keras.convolutional.py, not tf.keras.convolutional.py
question: 1, different channel use the same mask, not good?
"""
from __future__ import absolute_import
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

from keras import backend as K
from keras import activations
from keras import initializers
from keras import regularizers
from keras import constraints
from keras.engine import Layer
from keras.engine import InputSpec
from keras.utils import conv_utils
from keras.legacy import interfaces
import numpy as np
#from keras.engine.topology import _to_snake_case,_is_all_none,_collect_previous_mask
#from keras.engine.topology import _collect_input_shape,Node,_object_list_uid,_to_list
#from six.moves import zip
#import warnings
#import copy
#import inspect

class mask_Conv(Layer):
    """Abstract nD convolution layer (private, used as implementation base).

    This layer creates a convolution kernel that is convolved
    with the layer input to produce a tensor of outputs.
    If `use_bias` is True, a bias vector is created and added to the outputs.
    Finally, if `activation` is not `None`,
    it is applied to the outputs as well.

    # Arguments
        rank: An integer, the rank of the convolution,
            e.g. "2" for 2D convolution.
        filters: Integer, the dimensionality of the output space
            (i.e. the number output of filters in the convolution).
        kernel_size: An integer or tuple/list of n integers, specifying the
            dimensions of the convolution window.
        strides: An integer or tuple/list of n integers,
            specifying the strides of the convolution.
            Specifying any stride value != 1 is incompatible with specifying
            any `dilation_rate` value != 1.
        padding: One of `"valid"` or `"same"` (case-insensitive).
        data_format: A string,
            one of `channels_last` (default) or `channels_first`.
            The ordering of the dimensions in the inputs.
            `channels_last` corresponds to inputs with shape
            `(batch, ..., channels)` while `channels_first` corresponds to
            inputs with shape `(batch, channels, ...)`.
            It defaults to the `image_data_format` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "channels_last".
        dilation_rate: An integer or tuple/list of n integers, specifying
            the dilation rate to use for dilated convolution.
            Currently, specifying any `dilation_rate` value != 1 is
            incompatible with specifying any `strides` value != 1.
        activation: Activation function to use
            (see [activations](../activations.md)).
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: `a(x) = x`).
        use_bias: Boolean, whether the layer uses a bias vector.
        kernel_initializer: Initializer for the `kernel` weights matrix
            (see [initializers](../initializers.md)).
        bias_initializer: Initializer for the bias vector
            (see [initializers](../initializers.md)).
        kernel_regularizer: Regularizer function applied to
            the `kernel` weights matrix
            (see [regularizer](../regularizers.md)).
        bias_regularizer: Regularizer function applied to the bias vector
            (see [regularizer](../regularizers.md)).
        activity_regularizer: Regularizer function applied to
            the output of the layer (its "activation").
            (see [regularizer](../regularizers.md)).
        kernel_constraint: Constraint function applied to the kernel matrix
            (see [constraints](../constraints.md)).
        bias_constraint: Constraint function applied to the bias vector
            (see [constraints](../constraints.md)).
    """
    # repeat: repeat number of mask # different with:number of mask
    # mode: {single,normal,mul,knn}
    '''take Conv2D example, i.t. rank=2. there're 4 modes to choose. 
    inputs2.shape=[btch,:,:,channel,filters].
      # mode="single": inputs2.shape=[btch,:,:,1,5], single channel, multiple filter
      ('single' is included by 'normal', so 'single' mode should be remove, only for learning)
      mode="normal": inputs2.shape=[btch,:,:,3,5], multiple channels, multiple filters
      mode="knn": inputs2.shape=[btch,:,:,3,5], multiple channels, multiple filters
    fix=True means: 
      no weight added with inputs2, 
      repeat*masks + self.kernel.shape[-1] == outputs.shape[-1]
    when mode=knn,fix=True, mix should be given:
      if mix=True: inputs2 multiply with inputs1, but kernel convolute with inputs1
      if mix=False:both inputs and kernel multiply with inputs1
    fixNum: specify the num of data-kernel, when fix=True, fixNum is needed.
    fullBias: True means, bias.size == output.size
    '''
    
    def __init__(self, rank,
                 filters,
                 kernel_size,                 
                 strides=1,
                 padding='valid',
                 fixNum=0,mode='normal',repeat=1,fix=False,mix=True,fullBias=False,
                 data_format=None,
                 dilation_rate=1,
                 activation=None,
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 bias_regularizer=None,
                 activity_regularizer=None,
                 kernel_constraint=None,
                 bias_constraint=None,
                 **kwargs):
        if data_format is None:
          data_format = K.image_data_format()
        super(mask_Conv, self).__init__(**kwargs)
        self.rank = rank
        self.filters = filters
        self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, 'kernel_size')
        self.fixNum = fixNum
        self.mode = mode
        self.repeat = repeat
        self.fix = fix
        self.mix = mix
        self.fullBias = fullBias
        self.strides = conv_utils.normalize_tuple(strides, rank, 'strides')
        self.padding = conv_utils.normalize_padding(padding)
        self.data_format = data_format
        self.dilation_rate = conv_utils.normalize_tuple(dilation_rate, rank, 'dilation_rate')
        self.activation = activations.get(activation)
        self.use_bias = use_bias
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer)
        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.bias_regularizer = regularizers.get(bias_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)
        self.kernel_constraint = constraints.get(kernel_constraint)
        self.bias_constraint = constraints.get(bias_constraint)
        self.input_spec = InputSpec(ndim=self.rank + 2)

    def build(self, input_shape):
        if self.data_format == 'channels_first':
            channel_axis = 1
        else:
            channel_axis = -1
        if input_shape[channel_axis] is None:
            raise ValueError('The channel dimension of the inputs '
                             'should be defined. Found `None`.')
        input_dim = input_shape[channel_axis]
        kernel_shape = self.kernel_size + (input_dim, self.filters)

        if self.fix and self.fixNum==0:
            raise ValueError('if fix=True, fixNum must be >0.')
        if not self.fix and self.fixNum>0:
            raise ValueError('if fix=False, fixNum must be 0.')
            
        if self.fullBias:
            self.bias_size = self.kernel_size + (self.filters+self.fixNum)
        else:
            self.bias_size = (self.filters+self.fixNum,)

        self.kernel = self.add_weight(shape=kernel_shape,
                                      initializer=self.kernel_initializer,
                                      name='kernel',
                                      regularizer=self.kernel_regularizer,
                                      constraint=self.kernel_constraint)
        if self.use_bias:
            self.bias = self.add_weight(shape=self.bias_size,
                                        initializer=self.bias_initializer,
                                        name='bias',
                                        regularizer=self.bias_regularizer,
                                        constraint=self.bias_constraint)
        else:
            self.bias = None
        # Set input spec.
        self.input_spec = InputSpec(ndim=self.rank + 2,
                                    axes={channel_axis: input_dim})
        self.built = True

    # inputs1: input of data
    # inputs2: input of mask
    def call(self,inputs1,inputs2=None,*args,**kwargs):
        #print(kwargs)
        #mode = kwargs['mode']
        #inputs2 = kwargs['inputs2']
        #repeat = kwargs['repeat']
        masks = inputs2.shape[-1]#number of mask      
        print(self.kernel.shape)
        print(inputs2.shape)
        print("tf.shape(inputs2):",K.tf.shape(inputs2))
        print(inputs1.shape)
        print("-----self.strides:",self.strides)
        # normalize to norm(0,0.05)
        #inputs2 = K.tf.Variable(np.ones((inputs2_.shape+[1])[:-1],dtype='float32'))
        #inputs2 = K.tf.Variable(np.ones(inputs2_.shape,dtype='float32'))
        #K.tf.assign(inputs2, K.tf.divide(inputs2_,20*K.std(inputs2_)))
        #
        if self.mode == "single":
            assert self.repeat*masks <= self.kernel.shape[-1]
            assert inputs2.shape[1:-2] == self.kernel.shape[:-2]
            if self.rank == 1:
                print("self.dilation_rate:",self.dilation_rate)
                print("get_shape:",inputs1.get_shape())
                print("ndims:",inputs1.get_shape().ndims)
                print("ndims:",inputs1[0].get_shape().ndims)
                batch_size = K.tf.shape(inputs1)[0]
                initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                def cond(btch, *args):
                    return K.tf.less(btch, batch_size)
                def body(btch,outputs):
                    tmp_kernel = []
                    for ic in range(self.kernel.shape[-2]):
                        for ifs in range(self.kernel.shape[-1]):
                            if ifs < self.repeat*masks:
                                tmp_kernel.append(self.kernel[:,ic,ifs] + inputs2[btch,:,0,ifs%masks])
                            else:
                                tmp_kernel.append(self.kernel[:,ic,ifs])
                    print(len(tmp_kernel))
                    print(tmp_kernel[0].shape)
                    tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel,axis=self.rank), shape=self.kernel.shape)
                    outputs_tmp = K.conv1d(
                        K.tf.reshape(inputs1[btch,:,:], shape=[1]+list(inputs1.shape[1:])),
                        tmp_kernel,
                        strides=self.strides[0],
                        padding=self.padding,
                        data_format=self.data_format,
                        dilation_rate=self.dilation_rate[0])
                    print("outputs_tmp.shape",outputs_tmp.shape)
                    outputs = outputs.write(btch, outputs_tmp)
                    return btch + 1, outputs
                btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs])
                outputs = outputs.concat()
                print("btch, n:",btch)
                print("outputs.shape",outputs.shape)
            if self.rank == 2:
                #outputs = []
                print("self.dilation_rate:",self.dilation_rate)
                print("get_shape:",inputs1.get_shape())
                print("ndims:",inputs1.get_shape().ndims)
                print("ndims:",inputs1[0].get_shape().ndims)
                batch_size = K.tf.shape(inputs1)[0]
                initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                def cond(btch, *args):
                    return K.tf.less(btch, batch_size)
    #            def body(btch):# 不知道为什么这样写不行，非得加上outputs参数才行
    #                tmp_kernel = []
    #                for ic in range(self.kernel.shape[-2]):
    #                    for ifs in range(self.kernel.shape[-1]):
    #                        if ifs < self.repeat*masks:
    #                            tmp_kernel.append(self.kernel[:,:,ic,ifs] + inputs2[btch,:,:,0,ifs%masks])
    #                        else:
    #                            tmp_kernel.append(self.kernel[:,:,ic,ifs])
    #                print(len(tmp_kernel))
    #                print(tmp_kernel[0].shape)
    #                tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel,axis=2), shape=self.kernel.shape)
    #                outputs_tmp = K.conv2d(
    #                    K.tf.reshape(inputs1[btch,:,:,:], shape=[1]+list(inputs1.shape[1:])),
    #                    tmp_kernel,
    #                    strides=self.strides,
    #                    padding=self.padding,
    #                    data_format=self.data_format,
    #                    dilation_rate=self.dilation_rate)
    #                print("outputs_tmp.shape",outputs_tmp.shape)
    #                outputs.append(outputs_tmp)
    #                print("outputs(append).shape",len(outputs))
    #                #return K.tf.add(btch, 1)
    #                return btch + 1
                def body(btch,outputs):
                    tmp_kernel = []
                    for ic in range(self.kernel.shape[-2]):
                        for ifs in range(self.kernel.shape[-1]):
                            if ifs < self.repeat*masks:
                                tmp_kernel.append(self.kernel[:,:,ic,ifs] + inputs2[btch,:,:,0,ifs%masks])
                            else:
                                tmp_kernel.append(self.kernel[:,:,ic,ifs])
                    print(len(tmp_kernel))
                    print(tmp_kernel[0].shape)
                    tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel,axis=self.rank), shape=self.kernel.shape)
                    outputs_tmp = K.conv2d(
                        K.tf.reshape(inputs1[btch,:,:,:], shape=[1]+list(inputs1.shape[1:])),
                        tmp_kernel,
                        strides=self.strides,
                        padding=self.padding,
                        data_format=self.data_format,
                        dilation_rate=self.dilation_rate)
                    print("outputs_tmp.shape",outputs_tmp.shape)
                    outputs = outputs.write(btch, outputs_tmp)
                    #outputs.append(outputs_tmp)
                    #print("outputs(append).shape",len(outputs))
                    #return K.tf.add(btch, 1)
                    return btch + 1, outputs
                btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs])
                #print("outputs.shape",len(outputs))
                outputs = outputs.concat()
                #outputs = outputs.stack()
                #outputs = K.tf.concat(outputs,axis=0)
                print("btch, n:",btch)
                print("outputs.shape",outputs.shape)
    #        # 使用固定batch的写法：建立模型时需要传入参数batch
    #        mode = kwargs['mode']
    #        inputs2 = kwargs['inputs2']
    #        repeat = kwargs['repeat']
    #        batch = kwargs['batch']
    #        print(self.kernel.shape)
    #        print(inputs2.shape)
    #        print(inputs1.shape)
    #        if self.mode is "single":
    #            assert inputs2.shape[1:] == self.kernel.shape[:-2]
    #        elif self.mode is "knn":
    #            assert inputs2.shape[1:] == inputs1.shape[1:-1]
    #        #tmp_kernel = K.tf.Variable([self.kernel.shape])
    #        #tmp_kernel = self.kernel
    #        #tmp_kernel = np.zeros(self.kernel.shape,dtype='float32')
    #        tmp_kernel = []
    #        if self.rank == 1:
    #            for ic in range(self.kernel.shape[-2]):
    #                for ifs in range(self.repeat):
    #                    tmp_kernel.append(self.kernel[:,:,ic,ifs] + inputs2)
    #            tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel), shape=self.kernel.shape)
    #            outputs = K.conv1d(
    #                inputs1,
    #                self.kernel,
    #                strides=self.strides[0],
    #                padding=self.padding,
    #                data_format=self.data_format,
    #                dilation_rate=self.dilation_rate[0])
    #        if self.rank == 2:
    #            outputs = []
    #            tmp = inputs1[1,:,:,:]
    #            print("get_shape:",inputs1.get_shape())
    #            print("ndims:",inputs1.get_shape().ndims)
    #            print("ndims:",inputs1[0].get_shape().ndims)
    #            #for btch in range(batch):
    #            for btch in range(K.tf.shape(inputs1)[0]):
    #                tmp_kernel = []
    #                for ic in range(self.kernel.shape[-2]):
    #                    print("btch:",btch)
    #                    for ifs in range(self.kernel.shape[-1]):
    #                        if ifs < self.repeat:
    #                            tmp_kernel.append(self.kernel[:,:,ic,ifs] + inputs2[btch,:,:])
    #                        else:
    #                            tmp_kernel.append(self.kernel[:,:,ic,ifs])
    #                        #tmp_kernel.append(K.tf.add(self.kernel[:,:,ic,ifs], inputs2))# also OK
    #                        #tmp_kernel[:,:,ic,ifs] = K.tf.assign(tmp_kernel[:,:,ic,ifs], self.kernel[:,:,ic,ifs] + inputs2)
    #                        #tmp = self.kernel[:,:,ic,ifs]
    #                        #tmp_kernel[:,:,ic,ifs] = np.array(self.kernel[:,:,ic,ifs])
    #                        #tmp_kernel[:,:,ic,ifs] = self.kernel[:,:,ic,ifs] + inputs2
    #                        #tf.assign(self.kernel[:,:,ic,ifs], self.kernel[:,:,ic,ifs] + inputs2)
    #                        #self.kernel[:,:,ic,ifs] = self.kernel[:,:,ic,ifs] + inputs2
    #                #print(inputs2[0,0].eval())
    #                #print(self.kernel[:,:,0,0])
    #                #tmp2 = self.kernel[:,:,:,:]
    #                #tmp2[:,:,0,0] = self.kernel[:,:,0,0]
    #                print(len(tmp_kernel))
    #                print(tmp_kernel[0].shape)
    #                tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel,axis=2), shape=self.kernel.shape)
    #                outputs_tmp = K.conv2d(
    #                    K.tf.reshape(inputs1[btch,:,:,:], shape=[1]+list(inputs1.shape[1:])),
    #                    tmp_kernel,
    #                    strides=self.strides,
    #                    padding=self.padding,
    #                    data_format=self.data_format,
    #                    dilation_rate=self.dilation_rate)
    #                print("outputs_tmp.shape",outputs_tmp.shape)
    #                outputs.append(outputs_tmp)
    #            print("outputs.shape",len(outputs))
    #            outputs = K.tf.concat(outputs,axis=0)
    #            print("outputs.shape",outputs.shape)
            if self.rank == 3:
                print("get_shape:",inputs1.get_shape())
                print("ndims:",inputs1.get_shape().ndims)
                print("ndims:",inputs1[0].get_shape().ndims)
                batch_size = K.tf.shape(inputs1)[0]
                initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                def cond(btch, *args):
                    return K.tf.less(btch, batch_size)
                def body(btch,outputs):
                    tmp_kernel = []
                    for ic in range(self.kernel.shape[-2]):
                        for ifs in range(self.kernel.shape[-1]):
                            if ifs < self.repeat*masks:
                                tmp_kernel.append(self.kernel[:,:,:,ic,ifs] + inputs2[btch,:,:,:,0,ifs%masks])
                            else:
                                tmp_kernel.append(self.kernel[:,:,:,ic,ifs])
                    print(len(tmp_kernel))
                    print(tmp_kernel[0].shape)
                    tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel,axis=self.rank), shape=self.kernel.shape)
                    outputs_tmp = K.conv3d(
                        K.tf.reshape(inputs1[btch,:,:,:,:], shape=[1]+list(inputs1.shape[1:])),
                        tmp_kernel,
                        strides=self.strides,
                        padding=self.padding,
                        data_format=self.data_format,
                        dilation_rate=self.dilation_rate)
                    print("outputs_tmp.shape",outputs_tmp.shape)
                    outputs = outputs.write(btch, outputs_tmp)
                    return btch + 1, outputs
                btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs])
                outputs = outputs.concat()
                print("btch, n:",btch)
                print("outputs.shape",outputs.shape)
        
        elif self.mode == "normal":# mask multiply with input1, no weights
            print(self.mode)
            assert inputs2.shape[1:-2] == self.kernel.shape[:-2]
            inDim = self.kernel.shape[-2] # channels of input
            if self.rank == 1:
                print("self.dilation_rate:",self.dilation_rate)
                print("get_shape:",inputs1.get_shape())
                print("ndims:",inputs1.get_shape().ndims)
                print("ndims:",inputs1[0].get_shape().ndims)
                batch_size = K.tf.shape(inputs1)[0]
                initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                def cond(btch, *args):
                    return K.tf.less(btch, batch_size)
                def body(btch,outputs):
                    tmp_kernel = []
                    for ifs in range(self.kernel.shape[-1]):
                        if self.fix:
                            if ifs < self.repeat*masks:
                                if inputs2.shape[-2] == 1 :# inputs2 has single channel
                                    tmp_kernel.append(K.tile(inputs2[btch,:,:,ifs%masks],[1,inDim]))
                                elif inputs2.shape[-2] == self.kernel.shape[-2]: # inputs2 has full channel
                                    tmp_kernel.append(inputs2[btch,:,:,ifs%masks])
                                else:
                                    raise ValueError('The channel dimension of the inputs2 '
                                                     'should be 1 or equal to inputs1\'s channel.')
                            else:
                                tmp_kernel.append(self.kernel[:,:,ifs-self.repeat*masks])
                        else:
                            if ifs < self.repeat*masks:
                                if inputs2.shape[-2] == 1 :# inputs2 has single channel
                                    tmp_kernel.append(self.kernel[:,:,ifs] + K.tile(inputs2[btch,:,:,ifs%masks],[1,inDim]))
                                elif inputs2.shape[-2] == self.kernel.shape[-2]: # inputs2 has full channel
                                    tmp_kernel.append(self.kernel[:,:,ifs] + inputs2[btch,:,:,ifs%masks])
                                else:
                                    raise ValueError('The channel dimension of the inputs2 '
                                                     'should be 1 or equal to inputs1\'s channel.')
                            else:
                                tmp_kernel.append(self.kernel[:,:,ifs-self.repeat*masks])
#                    for ic in range(self.kernel.shape[-2]):
#                        for ifs in range(self.kernel.shape[-1]):
#                            if ifs < self.repeat*masks:
#                                tmp_kernel.append(inputs2[btch,:,ifs%masks])
#                            else:
#                                tmp_kernel.append(self.kernel[:,ic,ifs-self.repeat*masks])
                    print(len(tmp_kernel))
                    print(tmp_kernel[0].shape)
                    tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel,axis=self.rank), shape=self.kernel.shape)
                    outputs_tmp = K.conv1d(
                        K.tf.reshape(inputs1[btch,:,:], shape=[1]+list(inputs1.shape[1:])),
                        tmp_kernel,
                        strides=self.strides[0],
                        padding=self.padding,
                        data_format=self.data_format,
                        dilation_rate=self.dilation_rate[0])
                    print("outputs_tmp.shape",outputs_tmp.shape)
                    outputs = outputs.write(btch, outputs_tmp)
                    return btch + 1, outputs
                btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs])
                outputs = outputs.concat()
                print("btch, n:",btch)
                print("outputs.shape",outputs.shape)
            if self.rank == 2:
                #outputs = []
                print("self.dilation_rate:",self.dilation_rate)
                print("get_shape:",inputs1.get_shape())
                print("ndims:",inputs1.get_shape().ndims)
                print("ndims:",inputs1[0].get_shape().ndims)
                batch_size = K.tf.shape(inputs1)[0]
                initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                def cond(btch, *args):
                    return K.tf.less(btch, batch_size)
                def body(btch,outputs):
                    tmp_kernel = []
                    #tmp_kernel = inputs2[btch,:,:,:] + self.kernel[:,:,ic,:]
                    for ifs in range(self.kernel.shape[-1]):
                        if self.fix:
                            if ifs < self.repeat*masks:
                                if inputs2.shape[-2] == 1: # inputs2 has single channel
                                    tmp_kernel.append(K.tile(inputs2[btch,:,:,:,ifs%masks],[1,1,inDim]))
                                elif inputs2.shape[-2] == self.kernel.shape[-2]: # inputs2 has full channel
                                    tmp_kernel.append(inputs2[btch,:,:,:,ifs%masks])
                                else:
                                    raise ValueError('The channel dimension of the inputs2 '
                                                     'should be 1 or equal to inputs1\'s channel.')
                            else:
                                tmp_kernel.append(self.kernel[:,:,:,ifs-self.repeat*masks])
                        else:
                            if ifs < self.repeat*masks:
                                if inputs2.shape[-2] == 1: # inputs2 has single channel
                                    # tmp_kernel.append(K.tile(inputs2[btch,:,:,0,ifs%masks],[1,1,1,inDim,1]))
                                    tmp_kernel.append(self.kernel[:,:,:,ifs] + K.tile(inputs2[btch,:,:,:,ifs%masks],[1,1,inDim]))
                                elif inputs2.shape[-2] == self.kernel.shape[-2]: # inputs2 has full channel
                                    tmp_kernel.append(self.kernel[:,:,:,ifs] + inputs2[btch,:,:,:,ifs%masks])
                                else:
                                    raise ValueError('The channel dimension of the inputs2 '
                                                     'should be 1 or equal to inputs1\'s channel.')
                            else:
                                tmp_kernel.append(self.kernel[:,:,:,ifs-self.repeat*masks])
#                    for ic in range(self.kernel.shape[-2]):
#                        for ifs in range(self.kernel.shape[-1]):
#                            if ifs < self.repeat*masks:
#                                tmp_kernel.append(inputs2[btch,:,:,ifs%masks])
#                            else:
#                                tmp_kernel.append(self.kernel[:,:,ic,ifs-self.repeat*masks])
                    print(len(tmp_kernel))
                    print(tmp_kernel[0].shape)
                    tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel,axis=self.rank), shape=self.kernel.shape)
                    outputs_tmp = K.conv2d(
                        K.tf.reshape(inputs1[btch,:,:,:], shape=[1]+list(inputs1.shape[1:])),
                        tmp_kernel,
                        strides=self.strides,
                        padding=self.padding,
                        data_format=self.data_format,
                        dilation_rate=self.dilation_rate)
                    print("outputs_tmp.shape",outputs_tmp.shape)
                    outputs = outputs.write(btch, outputs_tmp)
                    #outputs.append(outputs_tmp)
                    #print("outputs(append).shape",len(outputs))
                    #return K.tf.add(btch, 1)
                    return btch + 1, outputs
                btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs])
                #print("outputs.shape",len(outputs))
                outputs = outputs.concat()
                #outputs = outputs.stack()
                #outputs = K.tf.concat(outputs,axis=0)
                print("btch, n:",btch)
                print("outputs.shape",outputs.shape)
            if self.rank == 3:
                print("get_shape:",inputs1.get_shape())
                print("ndims:",inputs1.get_shape().ndims)
                print("ndims:",inputs1[0].get_shape().ndims)
                batch_size = K.tf.shape(inputs1)[0]
                initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                def cond(btch, *args):
                    return K.tf.less(btch, batch_size)
                def body(btch,outputs):
                    tmp_kernel = []
                    for ifs in range(self.kernel.shape[-1]):
                        if self.fix:
                            if ifs < self.repeat*masks:
                                if inputs2.shape[-2] == 1: # inputs2 has single channel
                                    tmp_kernel.append(K.tile(inputs2[btch,:,:,:,:,ifs%masks],[1,1,1,inDim]))
                                elif inputs2.shape[-2] == self.kernel.shape[-2]: # inputs2 has full channel
                                    tmp_kernel.append(inputs2[btch,:,:,:,:,ifs%masks])
                                else:
                                    raise ValueError('The channel dimension of the inputs2 '
                                                     'should be 1 or equal to inputs1\'s channel.')
                            else:
                                tmp_kernel.append(self.kernel[:,:,:,:,ifs-self.repeat*masks])
                        else:                                    
                            if ifs < self.repeat*masks:
                                if inputs2.shape[-2] == 1: # inputs2 has single channel
                                    tmp_kernel.append(self.kernel[:,:,:,:,ifs] + K.tile(inputs2[btch,:,:,:,:,ifs%masks],[1,1,1,inDim]))
                                elif inputs2.shape[-2] == self.kernel.shape[-2]: # inputs2 has full channel
                                    tmp_kernel.append(self.kernel[:,:,:,:,ifs] + inputs2[btch,:,:,:,:,ifs%masks])
                                else:
                                    raise ValueError('The channel dimension of the inputs2 '
                                                     'should be 1 or equal to inputs1\'s channel.')
                            else:
                                tmp_kernel.append(self.kernel[:,:,:,:,ifs-self.repeat*masks])
#                    for ic in range(self.kernel.shape[-2]):
#                        for ifs in range(self.kernel.shape[-1]):
#                            if ifs < self.repeat*masks:
#                                tmp_kernel.append(self.kernel[:,:,:,ic,ifs] + inputs2[btch,:,:,:,ifs%masks])
#                            else:
#                                tmp_kernel.append(self.kernel[:,:,:,ic,ifs])
                    print(len(tmp_kernel))
                    print(tmp_kernel[0].shape)
                    tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel,axis=self.rank), shape=self.kernel.shape)
                    outputs_tmp = K.conv3d(
                        K.tf.reshape(inputs1[btch,:,:,:,:], shape=[1]+list(inputs1.shape[1:])),
                        tmp_kernel,
                        strides=self.strides,
                        padding=self.padding,
                        data_format=self.data_format,
                        dilation_rate=self.dilation_rate)
                    print("outputs_tmp.shape",outputs_tmp.shape)
                    outputs = outputs.write(btch, outputs_tmp)
                    return btch + 1, outputs
                btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs])
                outputs = outputs.concat()
                print("btch, n:",btch)
                print("outputs.shape",outputs.shape)
            assert self.kernel.shape[-1] == outputs.shape[-1]
                
        elif self.mode == "mul":# mask multiply with input1, no weights
            assert inputs2.shape[1:-2] == self.kernel.shape[:-2]
            if self.rank == 1:
                print("self.dilation_rate:",self.dilation_rate)
                print("inputs1.get_shape:",inputs1.get_shape())
                print("ndims:",inputs1.get_shape().ndims)
                print("ndims:",inputs1[0].get_shape().ndims)
                batch_size = K.tf.shape(inputs1)[0]
                initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                def cond(btch, *args):
                    return K.tf.less(btch, batch_size)
                def body(btch,outputs):
                    tmp_kernel = []
                    for ic in range(self.kernel.shape[-2]):
                        for ifs in range(self.kernel.shape[-1]):
                            if ifs < self.repeat*masks:
                                tmp_kernel.append(inputs2[btch,:,ifs%masks])
                            else:
                                tmp_kernel.append(self.kernel[:,ic,ifs-self.repeat*masks])
                    print(len(tmp_kernel))
                    print(tmp_kernel[0].shape)
                    tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel,axis=self.rank), shape=self.kernel.shape)
                    outputs_tmp = K.conv1d(
                        K.tf.reshape(inputs1[btch,:,:], shape=[1]+list(inputs1.shape[1:])),
                        tmp_kernel,
                        strides=self.strides[0],
                        padding=self.padding,
                        data_format=self.data_format,
                        dilation_rate=self.dilation_rate[0])
                    print("outputs_tmp.shape",outputs_tmp.shape)
                    outputs = outputs.write(btch, outputs_tmp)
                    return btch + 1, outputs
                btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs])
                outputs = outputs.concat()
                print("btch, n:",btch)
                print("outputs.shape",outputs.shape)
            if self.rank == 2:
                #outputs = []
                print("self.dilation_rate:",self.dilation_rate)
                print("get_shape:",inputs1.get_shape())
                print("ndims:",inputs1.get_shape().ndims)
                print("ndims:",inputs1[0].get_shape().ndims)
                batch_size = K.tf.shape(inputs1)[0]
                initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                def cond(btch, *args):
                    return K.tf.less(btch, batch_size)
                def body(btch,outputs):
                    tmp_kernel = []
                    #tmp_kernel = inputs2[btch,:,:,:] + self.kernel[:,:,ic,:]
                    for ic in range(self.kernel.shape[-2]):
                        for ifs in range(self.kernel.shape[-1]):
                            if ifs < self.repeat*masks:
                                tmp_kernel.append(inputs2[btch,:,:,ifs%masks])
                            else:
                                tmp_kernel.append(self.kernel[:,:,ic,ifs-self.repeat*masks])
                    print(len(tmp_kernel))
                    print(tmp_kernel[0].shape)
                    tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel,axis=self.rank), shape=self.kernel.shape)
                    outputs_tmp = K.conv2d(
                        K.tf.reshape(inputs1[btch,:,:,:], shape=[1]+list(inputs1.shape[1:])),
                        tmp_kernel,
                        strides=self.strides,
                        padding=self.padding,
                        data_format=self.data_format,
                        dilation_rate=self.dilation_rate)
                    print("outputs_tmp.shape",outputs_tmp.shape)
                    outputs = outputs.write(btch, outputs_tmp)
                    #outputs.append(outputs_tmp)
                    #print("outputs(append).shape",len(outputs))
                    #return K.tf.add(btch, 1)
                    return btch + 1, outputs
                btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs])
                #print("outputs.shape",len(outputs))
                outputs = outputs.concat()
                #outputs = outputs.stack()
                #outputs = K.tf.concat(outputs,axis=0)
                print("btch, n:",btch)
                print("outputs.shape",outputs.shape)
            if self.rank == 3:
                print("get_shape:",inputs1.get_shape())
                print("ndims:",inputs1.get_shape().ndims)
                print("ndims:",inputs1[0].get_shape().ndims)
                batch_size = K.tf.shape(inputs1)[0]
                initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                def cond(btch, *args):
                    return K.tf.less(btch, batch_size)
                def body(btch,outputs):
                    tmp_kernel = []
                    for ic in range(self.kernel.shape[-2]):
                        for ifs in range(self.kernel.shape[-1]):
                            if ifs < self.repeat*masks:
                                tmp_kernel.append(self.kernel[:,:,:,ic,ifs] + inputs2[btch,:,:,:,ifs%masks])
                            else:
                                tmp_kernel.append(self.kernel[:,:,:,ic,ifs])
                    print(len(tmp_kernel))
                    print(tmp_kernel[0].shape)
                    tmp_kernel = K.tf.reshape(K.tf.stack(tmp_kernel,axis=self.rank), shape=self.kernel.shape)
                    outputs_tmp = K.conv3d(
                        K.tf.reshape(inputs1[btch,:,:,:,:], shape=[1]+list(inputs1.shape[1:])),
                        tmp_kernel,
                        strides=self.strides,
                        padding=self.padding,
                        data_format=self.data_format,
                        dilation_rate=self.dilation_rate)
                    print("outputs_tmp.shape",outputs_tmp.shape)
                    outputs = outputs.write(btch, outputs_tmp)
                    return btch + 1, outputs
                btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs])
                outputs = outputs.concat()
                print("btch, n:",btch)
                print("outputs.shape",outputs.shape)
            assert self.repeat*masks + self.kernel.shape[-1] == outputs.shape[-1]
                
        elif self.mode == "knn":
            assert self.repeat*masks <= self.kernel.shape[-1]
            print(inputs1.shape)
            print(inputs2.shape)
            assert inputs2.shape[1:-2] == inputs1.shape[1:-1]#(None,10,10,1,8)vs(None,10,10,16)
            if self.fix:
                if self.mix:                 
                    if self.rank == 1:
                        outputs1 = K.conv1d(
                            inputs1,    # [32*9*9*3]
                            self.kernel,# [50*1*9*9*3]
                            strides=self.strides[0],
                            padding=self.padding,
                            data_format=self.data_format,
                            dilation_rate=self.dilation_rate[0])
                    if self.rank == 2:
                        outputs1 = K.conv2d(
                            inputs1,
                            self.kernel,
                            strides=self.strides,
                            padding=self.padding,
                            data_format=self.data_format,
                            dilation_rate=self.dilation_rate)
                    if self.rank == 3:
                        outputs1 = K.conv3d(
                            inputs1,
                            self.kernel,
                            strides=self.strides,
                            padding=self.padding,
                            data_format=self.data_format,
                            dilation_rate=self.dilation_rate)  
                    inputs1 = K.expand_dims(inputs1, axis=1)                     
                    perm = list(range(1,len(inputs2.shape)))
                    inputs2 = K.tf.transpose(inputs2, [0,perm.pop(-1)]+perm)
                    outputs2 = inputs1 * inputs2 # [50*1*9*9*3] vs [50*16*9*9*3]
                    print("outputs1.shape:  +++ ", outputs1.shape)
                    print("outputs2.shape:  +++ ", outputs2.shape)
                    outputs2 = K.sum(outputs2, axis=-1) # sum of different channels
                    print("outputs2.shape:  +++ ", outputs2.shape)
                    perm = list(range(2,len(outputs2.shape)))
                    outputs2 = K.tf.transpose(outputs2, [0]+perm+[1])
                    print("outputs2.shape:  +++ ", outputs2.shape)
                    outputs = K.tf.concat([outputs1, outputs2], -1) # [50*16*9*9*3] vs [50*32*9*9*3]
                    perm = list(range(2,len(outputs.shape)))
                else:
                    inputs1 = K.expand_dims(inputs1, axis=1)
                    
                    perm = list(range(len(self.kernel.shape)))
                    self.kernel = K.tf.transpose(self.kernel, [perm.pop(-1)]+perm)
                    outputs1 = inputs1 * self.kernel # [50*1*9*9*3] vs [32*9*9*3] batch=50,channel=3,filters=16,2DmapShape=9*9
                    
                    perm = list(range(1,len(inputs2.shape)))
                    inputs2 = K.tf.transpose(inputs2, [0,perm.pop(-1)]+perm)
                    outputs2 = inputs1 * inputs2 # [50*1*9*9*3] vs [50*16*9*9*3]
                    
                    outputs = K.tf.concat([outputs1, outputs2], 1) # [50*16*9*9*3] vs [50*32*9*9*3]
                    outputs = K.sum(outputs,axis=-1) # sum of different channels
                    perm = list(range(2,len(outputs.shape)))
                    outputs = K.tf.transpose(outputs, [0]+perm+[1])
                    
            else:            
                if self.rank == 1:
                    print("get_shape:",inputs1.get_shape())
                    print("ndims:",inputs1.get_shape().ndims)
                    print("ndims:",inputs1[0].get_shape().ndims)
                    batch_size = K.tf.shape(inputs1)[0]
                    initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                    ones_kernel = K.tf.Variable(np.ones(list(self.kernel.shape[:-1])+[1],dtype='float32'))                
                    ones_kernel = K.tf.convert_to_tensor(ones_kernel, dtype=K.tf.float32)
                    def cond(btch, *args):
                        return K.tf.less(btch, batch_size)
                    def body(btch,outputs):
                        output_bth = []
                        for ifs in range(self.kernel.shape[-1]):
                            if ifs < self.repeat*masks:
                                tmp_kernel = inputs1[btch,:,:] * K.tf.reshape(inputs2[btch,:,:,ifs%masks],shape=inputs2.shape[1:-1])
                                #tmp_kernel = inputs1[btch,:,:] * K.tf.transpose(inputs2[btch,:,:,ifs%masks],shape=inputs2.shape[1:-1])
                                output_bth_ifs1 = K.conv1d(
                                    K.tf.reshape(tmp_kernel, shape=[1]+list(tmp_kernel.shape)),
                                    ones_kernel,
                                    strides=self.strides[0],
                                    padding=self.padding,
                                    data_format=self.data_format,
                                    dilation_rate=self.dilation_rate[0])
                                output_bth_ifs2 = K.conv1d(
                                    K.tf.reshape(inputs1[btch,:,:], shape=[1]+list(inputs1.shape[1:])),
                                    K.tf.reshape(self.kernel[:,:,ifs], shape=list(self.kernel.shape[:-1])+[1]),
                                    strides=self.strides[0],
                                    padding=self.padding,
                                    data_format=self.data_format,
                                    dilation_rate=self.dilation_rate[0])
                                output_bth_ifs = output_bth_ifs1 + output_bth_ifs2
                            else:
                                output_bth_ifs = K.conv1d(
                                    K.tf.reshape(inputs1[btch,:,:], shape=[1]+list(inputs1.shape[1:])),
                                    K.tf.reshape(self.kernel[:,:,ifs], shape=list(self.kernel.shape[:-1])+[1]),
                                    strides=self.strides[0],
                                    padding=self.padding,
                                    data_format=self.data_format,
                                    dilation_rate=self.dilation_rate[0])
                            #if ifs == 0:
                            #    output_bth = output_bth_ifs
                            #else:
                            #    output_bth = output_bth + output_bth_ifs
                            output_bth.append(output_bth_ifs)
                        output_bth = K.tf.reshape(K.tf.stack(output_bth,axis=self.rank+1),shape=list(output_bth[0].shape[:-1])+[int(self.kernel.shape[-1])])
                        print("output_bth.shape",output_bth.shape)
                        outputs = outputs.write(btch, output_bth)
                        return btch + 1, outputs
                    btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs])
                    outputs = outputs.concat()
                    print("btch, n:",btch)
                    print("outputs.shape",outputs.shape)
                if self.rank == 2:
                    print("get_shape:",inputs1.get_shape())
                    print("ndims:",inputs1.get_shape().ndims)
                    print("ndims:",inputs1[0].get_shape().ndims)
                    batch_size = K.tf.shape(inputs1)[0]
                    initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                    ones_kernel = K.tf.Variable(np.ones(list(self.kernel.shape[:-1])+[1],dtype='float32'))
                    ones_kernel = K.tf.convert_to_tensor(ones_kernel, dtype=K.tf.float32)
                    def cond(btch, *args):
                        return K.tf.less(btch, batch_size)
                    def body(btch,outputs):
                        output_bth = []
                        for ifs in range(self.kernel.shape[-1]):
                            if ifs < self.repeat*masks:#inputs1:(None,10,10,16) vs inputs2(None,10,10,2)
                                tmp_kernel = inputs1[btch,:,:,:] * K.tf.reshape(inputs2[btch,:,:,:,ifs%masks],shape=inputs2.shape[1:-1])
                                output_bth_ifs1 = K.conv2d(
                                    K.tf.reshape(tmp_kernel, shape=[1]+list(tmp_kernel.shape)),
                                    ones_kernel,
                                    strides=self.strides,
                                    padding=self.padding,
                                    data_format=self.data_format,
                                    dilation_rate=self.dilation_rate)
                                output_bth_ifs2 = K.conv2d(
                                    K.tf.reshape(inputs1[btch,:,:,:], shape=[1]+list(inputs1.shape[1:])),
                                    K.tf.reshape(self.kernel[:,:,:,ifs], shape=list(self.kernel.shape[:-1])+[1]),
                                    strides=self.strides,
                                    padding=self.padding,
                                    data_format=self.data_format,
                                    dilation_rate=self.dilation_rate)
                                output_bth_ifs = output_bth_ifs1 + output_bth_ifs2
                            else:
                                output_bth_ifs = K.conv2d(
                                    K.tf.reshape(inputs1[btch,:,:,:], shape=[1]+list(inputs1.shape[1:])),
                                    K.tf.reshape(self.kernel[:,:,:,ifs], shape=list(self.kernel.shape[:-1])+[1]),
                                    strides=self.strides,
                                    padding=self.padding,
                                    data_format=self.data_format,
                                    dilation_rate=self.dilation_rate)
                            #if ifs == 0:
                            #    output_bth = output_bth_ifs
                            #else:
                            #    output_bth = output_bth + output_bth_ifs
                            output_bth.append(output_bth_ifs)
                        output_bth = K.tf.reshape(K.tf.stack(output_bth,axis=self.rank+1),shape=list(output_bth[0].shape[:-1])+[int(self.kernel.shape[-1])])
                        print("output_bth.shape",output_bth.shape)
                        outputs = outputs.write(btch, output_bth)
                        return btch + 1, outputs
                    btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs]) # type(outputs)=TensorArray
                    outputs = outputs.concat()
                    print("btch, n:",btch)
                    print("outputs.shape",outputs.shape)
                if self.rank == 3:
                    print("get_shape:",inputs1.get_shape())
                    print("ndims:",inputs1.get_shape().ndims)
                    print("ndims:",inputs1[0].get_shape().ndims)
                    batch_size = K.tf.shape(inputs1)[0]
                    initial_outputs = K.tf.TensorArray(dtype=K.tf.float32, size=batch_size)
                    print(self.kernel.shape)
                    print(type(self.kernel.shape))
                    ones_kernel = K.tf.Variable(np.ones(list(self.kernel.shape[:-1])+[1],dtype='float32'))
                    ones_kernel = K.tf.convert_to_tensor(ones_kernel, dtype=K.tf.float32)
                    def cond(btch, *args):
                        return K.tf.less(btch, batch_size)
                    def body(btch,outputs):
                        output_bth = []
                        #output_bth = K.tf.TensorArray(dtype=K.tf.float32, size=self.kernel.shape[-1])
                        for ifs in range(self.kernel.shape[-1]):
                            if ifs < self.repeat*masks:
                                #tmp_kernel = self.inputs1[btch,:,:,:] * inputs2[btch,:,:]
                                tmp_kernel = inputs1[btch,:,:,:,:] * K.tf.reshape(inputs2[btch,:,:,:,:,ifs%masks],shape=inputs2.shape[1:-1])
                                print("inputs2[btch,:,:,:]:",inputs2[btch,:,:,:].shape)
                                print("inputs1[btch,:,:,:,:]:",inputs1[btch,:,:,:,:].shape)
                                print("tmp_kernel.shape:",tmp_kernel.shape)
                                output_bth_ifs1 = K.conv3d(
                                    K.tf.reshape(tmp_kernel, shape=[1]+list(tmp_kernel.shape)),
                                    ones_kernel,
                                    strides=self.strides,
                                    padding=self.padding,
                                    data_format=self.data_format,
                                    dilation_rate=self.dilation_rate)
                                output_bth_ifs2 = K.conv3d(
                                    K.tf.reshape(inputs1[btch,:,:,:,:], shape=[1]+list(inputs1.shape[1:])),
                                    K.tf.reshape(self.kernel[:,:,:,:,ifs], shape=list(self.kernel.shape[:-1])+[1]),
                                    strides=self.strides,
                                    padding=self.padding,
                                    data_format=self.data_format,
                                    dilation_rate=self.dilation_rate)
                                output_bth_ifs = output_bth_ifs1 + output_bth_ifs2
                            else:
                                output_bth_ifs = K.conv3d(
                                    K.tf.reshape(inputs1[btch,:,:,:,:], shape=[1]+list(inputs1.shape[1:])),
                                    K.tf.reshape(self.kernel[:,:,:,:,ifs], shape=list(self.kernel.shape[:-1])+[1]),
                                    strides=self.strides,
                                    padding=self.padding,
                                    data_format=self.data_format,
                                    dilation_rate=self.dilation_rate)
                            #output_bth = output_bth.write(ifs, output_bth_ifs)
                            #if ifs == 0:
                            #    output_bth = output_bth_ifs
                            #else:
                            #    output_bth = output_bth + output_bth_ifs
                            output_bth.append(output_bth_ifs)
                        output_bth = K.tf.reshape(K.tf.stack(output_bth,axis=self.rank+1),shape=list(output_bth[0].shape[:-1])+[int(self.kernel.shape[-1])])
                        #output_bth = output_bth.concat()
                        #output_bth = K.tf.reduce_sum(output_bth, axis=0)
                        #output_bth = K.tf.reshape(output_bth, shape=[1]+list(output_bth.shape))
                        print("output_bth.shape", output_bth.shape)
                        outputs = outputs.write(btch, output_bth)
                        return btch + 1, outputs
                    btch, outputs = K.tf.while_loop(cond, body, [0, initial_outputs])
                    outputs = outputs.concat()
                    print("btch, n:",btch)
                    print("outputs.shape",outputs.shape)
        print("self.rank: ",self.rank)
        if self.use_bias:
            print("self.bias.shape: ",self.bias.shape)
        print("self.use_bias: ",self.use_bias)
        if self.use_bias:
            outputs = K.bias_add(
                outputs,
                self.bias,
                data_format=self.data_format)

        if self.activation is not None:
            return self.activation(outputs)
        print("-------------outputs.shape: ",outputs.shape)
        if self.fix:
            assert self.repeat*masks + self.kernel.shape[-1] == outputs.shape[-1]
        return outputs

    def compute_output_shape(self, input_shape):
        if self.data_format == 'channels_last':
            space = input_shape[1:-1]
            new_space = []
            for i in range(len(space)):
                new_dim = conv_utils.conv_output_length(
                    space[i],
                    self.kernel_size[i],
                    padding=self.padding,
                    stride=self.strides[i],
                    dilation=self.dilation_rate[i])
                new_space.append(new_dim)
            return (input_shape[0],) + tuple(new_space) + (self.filters+self.fixNum,)
        if self.data_format == 'channels_first':
            space = input_shape[2:]
            new_space = []
            for i in range(len(space)):
                new_dim = conv_utils.conv_output_length(
                    space[i],
                    self.kernel_size[i],
                    padding=self.padding,
                    stride=self.strides[i],
                    dilation=self.dilation_rate[i])
                new_space.append(new_dim)
            return (input_shape[0], self.filters+self.fixNum) + tuple(new_space)

    def get_config(self):
        config = {
            'rank': self.rank,
            'filters': self.filters,
            'kernel_size': self.kernel_size,
            'strides': self.strides,
            'padding': self.padding,
            'data_format': self.data_format,
            'dilation_rate': self.dilation_rate,
            'activation': activations.serialize(self.activation),
            'use_bias': self.use_bias,
            'kernel_initializer': initializers.serialize(self.kernel_initializer),
            'bias_initializer': initializers.serialize(self.bias_initializer),
            'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
            'bias_regularizer': regularizers.serialize(self.bias_regularizer),
            'activity_regularizer': regularizers.serialize(self.activity_regularizer),
            'kernel_constraint': constraints.serialize(self.kernel_constraint),
            'bias_constraint': constraints.serialize(self.bias_constraint)
        }
        base_config = super(mask_Conv, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

class mask_Conv1D(mask_Conv):
    """1D convolution layer (e.g. temporal convolution).

    This layer creates a convolution kernel that is convolved
    with the layer input over a single spatial (or temporal) dimension
    to produce a tensor of outputs.
    If `use_bias` is True, a bias vector is created and added to the outputs.
    Finally, if `activation` is not `None`,
    it is applied to the outputs as well.

    When using this layer as the first layer in a model,
    provide an `input_shape` argument
    (tuple of integers or `None`, e.g.
    `(10, 128)` for sequences of 10 vectors of 128-dimensional vectors,
    or `(None, 128)` for variable-length sequences of 128-dimensional vectors.

    # Arguments
        filters: Integer, the dimensionality of the output space
            (i.e. the number output of filters in the convolution).
        kernel_size: An integer or tuple/list of a single integer,
            specifying the length of the 1D convolution window.
        strides: An integer or tuple/list of a single integer,
            specifying the stride length of the convolution.
            Specifying any stride value != 1 is incompatible with specifying
            any `dilation_rate` value != 1.
        padding: One of `"valid"`, `"causal"` or `"same"` (case-insensitive).
            `"valid"` means "no padding".
            `"same"` results in padding the input such that
            the output has the same length as the original input.
            `"causal"` results in causal (dilated) convolutions, e.g. output[t]
            does not depend on input[t+1:]. Useful when modeling temporal data
            where the model should not violate the temporal order.
            See [WaveNet: A Generative Model for Raw Audio, section 2.1](https://arxiv.org/abs/1609.03499).
        dilation_rate: an integer or tuple/list of a single integer, specifying
            the dilation rate to use for dilated convolution.
            Currently, specifying any `dilation_rate` value != 1 is
            incompatible with specifying any `strides` value != 1.
        activation: Activation function to use
            (see [activations](../activations.md)).
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: `a(x) = x`).
        use_bias: Boolean, whether the layer uses a bias vector.
        kernel_initializer: Initializer for the `kernel` weights matrix
            (see [initializers](../initializers.md)).
        bias_initializer: Initializer for the bias vector
            (see [initializers](../initializers.md)).
        kernel_regularizer: Regularizer function applied to
            the `kernel` weights matrix
            (see [regularizer](../regularizers.md)).
        bias_regularizer: Regularizer function applied to the bias vector
            (see [regularizer](../regularizers.md)).
        activity_regularizer: Regularizer function applied to
            the output of the layer (its "activation").
            (see [regularizer](../regularizers.md)).
        kernel_constraint: Constraint function applied to the kernel matrix
            (see [constraints](../constraints.md)).
        bias_constraint: Constraint function applied to the bias vector
            (see [constraints](../constraints.md)).

    # Input shape
        3D tensor with shape: `(batch_size, steps, input_dim)`

    # Output shape
        3D tensor with shape: `(batch_size, new_steps, filters)`
        `steps` value might have changed due to padding or strides.
    """

    @interfaces.legacy_conv1d_support
    def __init__(self, filters,
                 kernel_size,
                 strides=1,
                 padding='valid',
                 dilation_rate=1,
                 activation=None,
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 bias_regularizer=None,
                 activity_regularizer=None,
                 kernel_constraint=None,
                 bias_constraint=None,
                 **kwargs):
        super(mask_Conv1D, self).__init__(
            rank=1,
            filters=filters,
            kernel_size=kernel_size,
            strides=strides,
            padding=padding,
            data_format='channels_last',
            dilation_rate=dilation_rate,
            activation=activation,
            use_bias=use_bias,
            kernel_initializer=kernel_initializer,
            bias_initializer=bias_initializer,
            kernel_regularizer=kernel_regularizer,
            bias_regularizer=bias_regularizer,
            activity_regularizer=activity_regularizer,
            kernel_constraint=kernel_constraint,
            bias_constraint=bias_constraint,
            **kwargs)
        self.input_spec = InputSpec(ndim=3)

    def get_config(self):
        config = super(mask_Conv1D, self).get_config()
        config.pop('rank')
        config.pop('data_format')
        return config

class mask_Conv2D(mask_Conv):
    """2D convolution layer (e.g. spatial convolution over images).

    This layer creates a convolution kernel that is convolved
    with the layer input to produce a tensor of
    outputs. If `use_bias` is True,
    a bias vector is created and added to the outputs. Finally, if
    `activation` is not `None`, it is applied to the outputs as well.

    When using this layer as the first layer in a model,
    provide the keyword argument `input_shape`
    (tuple of integers, does not include the sample axis),
    e.g. `input_shape=(128, 128, 3)` for 128x128 RGB pictures
    in `data_format="channels_last"`.

    # Arguments
        filters: Integer, the dimensionality of the output space
            (i.e. the number output of filters in the convolution).
        kernel_size: An integer or tuple/list of 2 integers, specifying the
            width and height of the 2D convolution window.
            Can be a single integer to specify the same value for
            all spatial dimensions.
        strides: An integer or tuple/list of 2 integers,
            specifying the strides of the convolution along the width and height.
            Can be a single integer to specify the same value for
            all spatial dimensions.
            Specifying any stride value != 1 is incompatible with specifying
            any `dilation_rate` value != 1.
        padding: one of `"valid"` or `"same"` (case-insensitive).
        data_format: A string,
            one of `channels_last` (default) or `channels_first`.
            The ordering of the dimensions in the inputs.
            `channels_last` corresponds to inputs with shape
            `(batch, height, width, channels)` while `channels_first`
            corresponds to inputs with shape
            `(batch, channels, height, width)`.
            It defaults to the `image_data_format` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "channels_last".
        dilation_rate: an integer or tuple/list of 2 integers, specifying
            the dilation rate to use for dilated convolution.
            Can be a single integer to specify the same value for
            all spatial dimensions.
            Currently, specifying any `dilation_rate` value != 1 is
            incompatible with specifying any stride value != 1.
        activation: Activation function to use
            (see [activations](../activations.md)).
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: `a(x) = x`).
        use_bias: Boolean, whether the layer uses a bias vector.
        kernel_initializer: Initializer for the `kernel` weights matrix
            (see [initializers](../initializers.md)).
        bias_initializer: Initializer for the bias vector
            (see [initializers](../initializers.md)).
        kernel_regularizer: Regularizer function applied to
            the `kernel` weights matrix
            (see [regularizer](../regularizers.md)).
        bias_regularizer: Regularizer function applied to the bias vector
            (see [regularizer](../regularizers.md)).
        activity_regularizer: Regularizer function applied to
            the output of the layer (its "activation").
            (see [regularizer](../regularizers.md)).
        kernel_constraint: Constraint function applied to the kernel matrix
            (see [constraints](../constraints.md)).
        bias_constraint: Constraint function applied to the bias vector
            (see [constraints](../constraints.md)).

    # Input shape
        4D tensor with shape:
        `(samples, channels, rows, cols)` if data_format='channels_first'
        or 4D tensor with shape:
        `(samples, rows, cols, channels)` if data_format='channels_last'.

    # Output shape
        4D tensor with shape:
        `(samples, filters, new_rows, new_cols)` if data_format='channels_first'
        or 4D tensor with shape:
        `(samples, new_rows, new_cols, filters)` if data_format='channels_last'.
        `rows` and `cols` values might have changed due to padding.
    """

    @interfaces.legacy_conv2d_support
    def __init__(self, filters,
                 kernel_size,
                 strides=(1, 1),
                 padding='valid',
                 data_format=None,
                 dilation_rate=(1, 1),
                 activation=None,
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 bias_regularizer=None,
                 activity_regularizer=None,
                 kernel_constraint=None,
                 bias_constraint=None,
                 **kwargs):
        super(mask_Conv2D, self).__init__(
            rank=2,
            filters=filters,
            kernel_size=kernel_size,
            strides=strides,
            padding=padding,
            data_format=data_format,
            dilation_rate=dilation_rate,
            activation=activation,
            use_bias=use_bias,
            kernel_initializer=kernel_initializer,
            bias_initializer=bias_initializer,
            kernel_regularizer=kernel_regularizer,
            bias_regularizer=bias_regularizer,
            activity_regularizer=activity_regularizer,
            kernel_constraint=kernel_constraint,
            bias_constraint=bias_constraint,
            **kwargs)
        self.input_spec = InputSpec(ndim=4)

    def get_config(self):
        config = super(mask_Conv2D, self).get_config()
        config.pop('rank')
        return config


class mask_Conv3D(mask_Conv):
    """3D convolution layer (e.g. spatial convolution over volumes).

    This layer creates a convolution kernel that is convolved
    with the layer input to produce a tensor of
    outputs. If `use_bias` is True,
    a bias vector is created and added to the outputs. Finally, if
    `activation` is not `None`, it is applied to the outputs as well.

    When using this layer as the first layer in a model,
    provide the keyword argument `input_shape`
    (tuple of integers, does not include the sample axis),
    e.g. `input_shape=(128, 128, 128, 1)` for 128x128x128 volumes
    with a single channel,
    in `data_format="channels_last"`.

    # Arguments
        filters: Integer, the dimensionality of the output space
            (i.e. the number output of filters in the convolution).
        kernel_size: An integer or tuple/list of 3 integers, specifying the
            depth, height and width of the 3D convolution window.
            Can be a single integer to specify the same value for
            all spatial dimensions.
        strides: An integer or tuple/list of 3 integers,
            specifying the strides of the convolution along each spatial dimension.
            Can be a single integer to specify the same value for
            all spatial dimensions.
            Specifying any stride value != 1 is incompatible with specifying
            any `dilation_rate` value != 1.
        padding: one of `"valid"` or `"same"` (case-insensitive).
        data_format: A string,
            one of `channels_last` (default) or `channels_first`.
            The ordering of the dimensions in the inputs.
            `channels_last` corresponds to inputs with shape
            `(batch, spatial_dim1, spatial_dim2, spatial_dim3, channels)`
            while `channels_first` corresponds to inputs with shape
            `(batch, channels, spatial_dim1, spatial_dim2, spatial_dim3)`.
            It defaults to the `image_data_format` value found in your
            Keras config file at `~/.keras/keras.json`.
            If you never set it, then it will be "channels_last".
        dilation_rate: an integer or tuple/list of 3 integers, specifying
            the dilation rate to use for dilated convolution.
            Can be a single integer to specify the same value for
            all spatial dimensions.
            Currently, specifying any `dilation_rate` value != 1 is
            incompatible with specifying any stride value != 1.
        activation: Activation function to use
            (see [activations](../activations.md)).
            If you don't specify anything, no activation is applied
            (ie. "linear" activation: `a(x) = x`).
        use_bias: Boolean, whether the layer uses a bias vector.
        kernel_initializer: Initializer for the `kernel` weights matrix
            (see [initializers](../initializers.md)).
        bias_initializer: Initializer for the bias vector
            (see [initializers](../initializers.md)).
        kernel_regularizer: Regularizer function applied to
            the `kernel` weights matrix
            (see [regularizer](../regularizers.md)).
        bias_regularizer: Regularizer function applied to the bias vector
            (see [regularizer](../regularizers.md)).
        activity_regularizer: Regularizer function applied to
            the output of the layer (its "activation").
            (see [regularizer](../regularizers.md)).
        kernel_constraint: Constraint function applied to the kernel matrix
            (see [constraints](../constraints.md)).
        bias_constraint: Constraint function applied to the bias vector
            (see [constraints](../constraints.md)).

    # Input shape
        5D tensor with shape:
        `(samples, channels, conv_dim1, conv_dim2, conv_dim3)` if data_format='channels_first'
        or 5D tensor with shape:
        `(samples, conv_dim1, conv_dim2, conv_dim3, channels)` if data_format='channels_last'.

    # Output shape
        5D tensor with shape:
        `(samples, filters, new_conv_dim1, new_conv_dim2, new_conv_dim3)` if data_format='channels_first'
        or 5D tensor with shape:
        `(samples, new_conv_dim1, new_conv_dim2, new_conv_dim3, filters)` if data_format='channels_last'.
        `new_conv_dim1`, `new_conv_dim2` and `new_conv_dim3` values might have changed due to padding.
    """

    @interfaces.legacy_conv3d_support
    def __init__(self, filters,
                 kernel_size,
                 strides=(1, 1, 1),
                 padding='valid',
                 data_format=None,
                 dilation_rate=(1, 1, 1),
                 activation=None,
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 bias_regularizer=None,
                 activity_regularizer=None,
                 kernel_constraint=None,
                 bias_constraint=None,
                 **kwargs):
        super(mask_Conv3D, self).__init__(
            rank=3,
            filters=filters,
            kernel_size=kernel_size,
            strides=strides,
            padding=padding,
            data_format=data_format,
            dilation_rate=dilation_rate,
            activation=activation,
            use_bias=use_bias,
            kernel_initializer=kernel_initializer,
            bias_initializer=bias_initializer,
            kernel_regularizer=kernel_regularizer,
            bias_regularizer=bias_regularizer,
            activity_regularizer=activity_regularizer,
            kernel_constraint=kernel_constraint,
            bias_constraint=bias_constraint,
            **kwargs)
        self.input_spec = InputSpec(ndim=5)

    def get_config(self):
        config = super(mask_Conv3D, self).get_config()
        config.pop('rank')
        return config

# product of kernel and input, multiply_conv
class mul_Conv(Layer):
    """ change from keras.convolutional.py, not tf.keras.convolutional.py
        kernel is multiple with input
    """

    def __init__(self, rank,
                 filters,                 
                 kernel_size,
                 data_format='channels_last', # only allow channels_last !
                 activation=None,
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 bias_regularizer=None,
                 activity_regularizer=None,
                 kernel_constraint=None,
                 bias_constraint=None,
                 **kwargs):
        super(mul_Conv, self).__init__(**kwargs)
        self.rank = rank
        self.filters = filters
        self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, 'kernel_size')
        self.data_format = 'channels_last'
        self.activation = activations.get(activation)
        self.use_bias = use_bias
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer)
        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.bias_regularizer = regularizers.get(bias_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)
        self.kernel_constraint = constraints.get(kernel_constraint)
        self.bias_constraint = constraints.get(bias_constraint)
        self.input_spec = InputSpec(ndim=self.rank + 2)

    def build(self, input_shape):
        #if self.data_format == 'channels_first':#'channels_first' is not allowed
        #    channel_axis = 1
        #    bias_shape = input_shape[2:] + self.filters
        channel_axis = -1
        if input_shape[channel_axis] is None:
            raise ValueError('The channel dimension of the inputs '
                             'should be defined. Found `None`.')
        input_dim = input_shape[channel_axis]
        kernel_shape = input_shape[1:] + (self.filters,)
        bias_shape = input_shape[1:-1] + (self.filters,) #type(input_shape) = tuple

        self.kernel = self.add_weight(shape=kernel_shape,
                                      initializer=self.kernel_initializer,
                                      name='kernel',
                                      regularizer=self.kernel_regularizer,
                                      constraint=self.kernel_constraint,
                                      trainable=True)
        if self.use_bias:
            self.bias = self.add_weight(shape=bias_shape,
                                        initializer=self.bias_initializer,
                                        name='bias',
                                        regularizer=self.bias_regularizer,
                                        constraint=self.bias_constraint)
        else:
            self.bias = None
        # Set input spec.
        self.input_spec = InputSpec(ndim=self.rank + 2,
                                    axes={channel_axis: input_dim})
        self.built = True

    def call(self, inputs):
        # move last-axis to 0-axis, so that tf.multiply can be used
        # equivalent-to:perm=[len(self.kernel.shape)]+list(range(len(self.kernel.shape)-1))
        perm = list(range(len(self.kernel.shape)))
        self.kernel = K.tf.transpose(self.kernel, [perm.pop(-1)]+perm)
        inputs = K.expand_dims(inputs, axis=1)
        # element-wise multiply, kernel has one more dimension(axis=1) than inputs
        outputs = K.tf.multiply(inputs,self.kernel)
        print("out:",K.shape(outputs), outputs.shape)
        outputs = K.sum(outputs,axis=-1) # sum of different channels
        # move 1-axis to last-axis, this dimension = filters
        perm = list(range(2,len(outputs.shape)))
        print("in:",K.shape(inputs), inputs.shape)
        print("knl:",K.shape(self.kernel), self.kernel.shape)
        print("out:",K.shape(outputs), outputs.shape)
        outputs = K.tf.transpose(outputs, [0]+perm+[1])
        print("out:",K.shape(outputs), outputs.shape)

        if self.use_bias:
            outputs = K.tf.add(outputs, self.bias)

        if self.activation is not None:
            return self.activation(outputs)
        return outputs

    def compute_output_shape(self, input_shape):
        return tuple(input_shape[:-1]) + (self.filters,)

    def get_config(self):
        config = {
            'rank': self.rank,
            'filters': self.filters,
            'kernel_size': self.kernel_size,
            'data_format': self.data_format,
            'activation': activations.serialize(self.activation),
            'use_bias': self.use_bias,
            'kernel_initializer': initializers.serialize(self.kernel_initializer),
            'bias_initializer': initializers.serialize(self.bias_initializer),
            'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
            'bias_regularizer': regularizers.serialize(self.bias_regularizer),
            'activity_regularizer': regularizers.serialize(self.activity_regularizer),
            'kernel_constraint': constraints.serialize(self.kernel_constraint),
            'bias_constraint': constraints.serialize(self.bias_constraint)
        }
        base_config = super(mask_Conv, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

class mul_Conv2D(mul_Conv):
    @interfaces.legacy_conv2d_support
    def __init__(self, filters,
                 kernel_size,
                 data_format=None,
                 activation=None,
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 bias_regularizer=None,
                 activity_regularizer=None,
                 kernel_constraint=None,
                 bias_constraint=None,
                 **kwargs):
        super(mul_Conv2D, self).__init__(
            rank=2,
            filters=filters,
            kernel_size=kernel_size,
            data_format=data_format,
            activation=activation,
            use_bias=use_bias,
            kernel_initializer=kernel_initializer,
            bias_initializer=bias_initializer,
            kernel_regularizer=kernel_regularizer,
            bias_regularizer=bias_regularizer,
            activity_regularizer=activity_regularizer,
            kernel_constraint=kernel_constraint,
            bias_constraint=bias_constraint,
            **kwargs)
        self.input_spec = InputSpec(ndim=4)

    def get_config(self):
        config = super(mul_Conv2D, self).get_config()
        config.pop('rank')
        return config

class fix_Conv(Layer):
    """ change from keras.convolutional.py, not tf.keras.convolutional.py
        kernel is not trainable
    """

    def __init__(self, rank,
                 filters,
                 kernel_size,
                 strides=1,
                 padding='valid',
                 data_format=None,
                 dilation_rate=1,
                 activation=None,
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 bias_regularizer=None,
                 activity_regularizer=None,
                 kernel_constraint=None,
                 bias_constraint=None,
                 **kwargs):
        if data_format is None:
          data_format = K.image_data_format()
        super(fix_Conv, self).__init__(**kwargs)
        self.rank = rank
        self.filters = filters
        self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, 'kernel_size')
        self.strides = conv_utils.normalize_tuple(strides, rank, 'strides')
        self.padding = conv_utils.normalize_padding(padding)
        self.data_format = data_format
        self.dilation_rate = conv_utils.normalize_tuple(dilation_rate, rank, 'dilation_rate')
        self.activation = activations.get(activation)
        self.use_bias = use_bias
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.bias_initializer = initializers.get(bias_initializer)
        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.bias_regularizer = regularizers.get(bias_regularizer)
        self.activity_regularizer = regularizers.get(activity_regularizer)
        self.kernel_constraint = constraints.get(kernel_constraint)
        self.bias_constraint = constraints.get(bias_constraint)
        self.input_spec = InputSpec(ndim=self.rank + 2)

    def build(self, input_shape):
        if self.data_format == 'channels_first':
            channel_axis = 1
        else:
            channel_axis = -1
        if input_shape[channel_axis] is None:
            raise ValueError('The channel dimension of the inputs '
                             'should be defined. Found `None`.')
        input_dim = input_shape[channel_axis]
        kernel_shape = self.kernel_size + (input_dim, self.filters)

        self.kernel = self.add_weight(shape=kernel_shape,
                                      initializer=self.kernel_initializer,
                                      name='kernel',
                                      regularizer=self.kernel_regularizer,
                                      constraint=self.kernel_constraint,
                                      trainable=False)
        if self.use_bias:
            self.bias = self.add_weight(shape=(self.filters,),
                                        initializer=self.bias_initializer,
                                        name='bias',
                                        regularizer=self.bias_regularizer,
                                        constraint=self.bias_constraint)
        else:
            self.bias = None
        # Set input spec.
        self.input_spec = InputSpec(ndim=self.rank + 2,
                                    axes={channel_axis: input_dim})
        self.built = True

    def call(self, inputs):
        if self.rank == 1:
            outputs = K.conv1d(
                inputs,
                self.kernel,
                strides=self.strides[0],
                padding=self.padding,
                data_format=self.data_format,
                dilation_rate=self.dilation_rate[0])
        if self.rank == 2:
            outputs = K.conv2d(
                inputs,
                self.kernel,
                strides=self.strides,
                padding=self.padding,
                data_format=self.data_format,
                dilation_rate=self.dilation_rate)
        if self.rank == 3:
            outputs = K.conv3d(
                inputs,
                self.kernel,
                strides=self.strides,
                padding=self.padding,
                data_format=self.data_format,
                dilation_rate=self.dilation_rate)

        if self.use_bias:
            outputs = K.bias_add(
                outputs,
                self.bias,
                data_format=self.data_format)

        if self.activation is not None:
            return self.activation(outputs)
        return outputs

    def compute_output_shape(self, input_shape):
        if self.data_format == 'channels_last':
            space = input_shape[1:-1]
            new_space = []
            for i in range(len(space)):
                new_dim = conv_utils.conv_output_length(
                    space[i],
                    self.kernel_size[i],
                    padding=self.padding,
                    stride=self.strides[i],
                    dilation=self.dilation_rate[i])
                new_space.append(new_dim)
            return (input_shape[0],) + tuple(new_space) + (self.filters,)
        if self.data_format == 'channels_first':
            space = input_shape[2:]
            new_space = []
            for i in range(len(space)):
                new_dim = conv_utils.conv_output_length(
                    space[i],
                    self.kernel_size[i],
                    padding=self.padding,
                    stride=self.strides[i],
                    dilation=self.dilation_rate[i])
                new_space.append(new_dim)
            return (input_shape[0], self.filters) + tuple(new_space)

    def get_config(self):
        config = {
            'rank': self.rank,
            'filters': self.filters,
            'kernel_size': self.kernel_size,
            'strides': self.strides,
            'padding': self.padding,
            'data_format': self.data_format,
            'dilation_rate': self.dilation_rate,
            'activation': activations.serialize(self.activation),
            'use_bias': self.use_bias,
            'kernel_initializer': initializers.serialize(self.kernel_initializer),
            'bias_initializer': initializers.serialize(self.bias_initializer),
            'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
            'bias_regularizer': regularizers.serialize(self.bias_regularizer),
            'activity_regularizer': regularizers.serialize(self.activity_regularizer),
            'kernel_constraint': constraints.serialize(self.kernel_constraint),
            'bias_constraint': constraints.serialize(self.bias_constraint)
        }
        base_config = super(mask_Conv, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

class fix_Conv1D(fix_Conv):
    @interfaces.legacy_conv1d_support
    def __init__(self, filters,
                 kernel_size,
                 strides=1,
                 padding='valid',
                 dilation_rate=1,
                 activation=None,
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 bias_regularizer=None,
                 activity_regularizer=None,
                 kernel_constraint=None,
                 bias_constraint=None,
                 **kwargs):
        super(fix_Conv1D, self).__init__(
            rank=1,
            filters=filters,
            kernel_size=kernel_size,
            strides=strides,
            padding=padding,
            data_format='channels_last',
            dilation_rate=dilation_rate,
            activation=activation,
            use_bias=use_bias,
            kernel_initializer=kernel_initializer,
            bias_initializer=bias_initializer,
            kernel_regularizer=kernel_regularizer,
            bias_regularizer=bias_regularizer,
            activity_regularizer=activity_regularizer,
            kernel_constraint=kernel_constraint,
            bias_constraint=bias_constraint,
            **kwargs)
        self.input_spec = InputSpec(ndim=3)

    def get_config(self):
        config = super(mask_Conv1D, self).get_config()
        config.pop('rank')
        config.pop('data_format')
        return config

class fix_Conv2D(fix_Conv):
    @interfaces.legacy_conv2d_support
    def __init__(self, filters,
                 kernel_size,
                 strides=(1, 1),
                 padding='valid',
                 data_format=None,
                 dilation_rate=(1, 1),
                 activation=None,
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 bias_regularizer=None,
                 activity_regularizer=None,
                 kernel_constraint=None,
                 bias_constraint=None,
                 **kwargs):
        super(fix_Conv2D, self).__init__(
            rank=2,
            filters=filters,
            kernel_size=kernel_size,
            strides=strides,
            padding=padding,
            data_format=data_format,
            dilation_rate=dilation_rate,
            activation=activation,
            use_bias=use_bias,
            kernel_initializer=kernel_initializer,
            bias_initializer=bias_initializer,
            kernel_regularizer=kernel_regularizer,
            bias_regularizer=bias_regularizer,
            activity_regularizer=activity_regularizer,
            kernel_constraint=kernel_constraint,
            bias_constraint=bias_constraint,
            **kwargs)
        self.input_spec = InputSpec(ndim=4)

    def get_config(self):
        config = super(mask_Conv2D, self).get_config()
        config.pop('rank')
        return config

class fix_Conv3D(fix_Conv):
    @interfaces.legacy_conv3d_support
    def __init__(self, filters,
                 kernel_size,
                 strides=(1, 1, 1),
                 padding='valid',
                 data_format=None,
                 dilation_rate=(1, 1, 1),
                 activation=None,
                 use_bias=True,
                 kernel_initializer='glorot_uniform',
                 bias_initializer='zeros',
                 kernel_regularizer=None,
                 bias_regularizer=None,
                 activity_regularizer=None,
                 kernel_constraint=None,
                 bias_constraint=None,
                 **kwargs):
        super(fix_Conv3D, self).__init__(
            rank=3,
            filters=filters,
            kernel_size=kernel_size,
            strides=strides,
            padding=padding,
            data_format=data_format,
            dilation_rate=dilation_rate,
            activation=activation,
            use_bias=use_bias,
            kernel_initializer=kernel_initializer,
            bias_initializer=bias_initializer,
            kernel_regularizer=kernel_regularizer,
            bias_regularizer=bias_regularizer,
            activity_regularizer=activity_regularizer,
            kernel_constraint=kernel_constraint,
            bias_constraint=bias_constraint,
            **kwargs)
        self.input_spec = InputSpec(ndim=5)

    def get_config(self):
        config = super(mask_Conv3D, self).get_config()
        config.pop('rank')
        return config

if __name__ == "__main__":
# 实现相乘的层
#    import numpy as np
#    import keras
#    import tensorflow as tf
#    import sys
#    class multi(Layer):
#        def __init__(self, output_dim=1, **kwargs):
#            self.output_dim = output_dim
#            super(multi, self).__init__(**kwargs)
#        def build(self, input_shape):
#            print(inputs.shape)
#            assert len(input_shape) >= 2
#            input_dim = input_shape[-1]
#            self.input_spec = InputSpec(min_ndim=2, axes={-1: input_dim})
#            self.built = True
#            super(multi, self).build(input_shape)
#        def call(self, inputs):
#            input1 = inputs[0]
#            input2 = inputs[1]
#            output = K.tf.multiply(input1,input2)
#            print(output.shape)
#            print(inputs.shape[-1])
#            print(inputs.shape)
#            return output
#        def compute_output_shape(self, input_shape):
#            return (input_shape[1:])
#    x1 = np.array([[1,2,3,4]])
#    x2 = np.array([[1,2,3,4]])
#    input1 = tf.constant(x1, dtype=tf.float32)
#    input2 = tf.constant(x2, dtype=tf.float32)
#    inputs = keras.layers.concatenate([input1, input2],axis=0)
#    multi_res = multi(1)(inputs)
#    with tf.Session() as s:
#        s.run(tf.global_variables_initializer())
#        o = s.run(multi_res)
#    sys.exit(0)
    
    #import numpy as np
    import tensorflow as tf
    import sys
    sys.exit(0)
    
    
    # test mul_Conv2D
    zeros_init = tf.constant_initializer(0)
    sim = np.ones((2,3,3))
    sim12 = tf.constant(sim, dtype=tf.float32)
    sim_knn = np.ones((2,5,5), dtype=np.float32)
    sim_knn = tf.constant(sim_knn, dtype=tf.float32)
    i = np.round(np.random.random((2, 5, 5, 2))*100)
    input = tf.constant(i, dtype=tf.float32)
    conv_norm2 = mul_Conv2D(1, kernel_size=(3, 3),kernel_initializer=zeros_init,bias_initializer=zeros_init,
                         activation='relu')(input)

    # test mask_Conv1D
    zeros_init = tf.constant_initializer(0)
    sim = np.ones((2,3))
    sim12 = tf.constant(sim, dtype=tf.float32)
    sim_knn = np.ones((2,5), dtype=np.float32)
    sim_knn = tf.constant(sim_knn, dtype=tf.float32)
    i = np.round(np.random.random((2, 5, 2))*100)
    input = tf.constant(i, dtype=tf.float32)    
    conv_norm1 = mask_Conv1D(1, kernel_size=(3),kernel_initializer=zeros_init,bias_initializer=zeros_init,
                         activation='relu')(input,inputs2=sim12,repeat=1,mode='normal')
    conv_knn1 = mask_Conv1D(1, kernel_size=(3),kernel_initializer=zeros_init,bias_initializer=zeros_init,
                         activation='relu')(input,inputs2=sim_knn,repeat=1,mode='knn')
    with tf.Session() as s:
        s.run(tf.global_variables_initializer())
        o = s.run(conv_norm1)
        o_knn = s.run(conv_knn1)
    t1=o[:,:,0]
    t=o_knn[:,:,0]
    t2=i[0,:,:]
    
    sys.exit(0)
    
    # test mask_Conv2D
    zeros_init = tf.constant_initializer(0)
    sim = np.ones((2,3,3))
    sim12 = tf.constant(sim, dtype=tf.float32)
    sim_knn = np.ones((2,5,5), dtype=np.float32)
    sim_knn = tf.constant(sim_knn, dtype=tf.float32)
    i = np.round(np.random.random((2, 5, 5, 2))*100)
    input = tf.constant(i, dtype=tf.float32)    
    conv_norm2 = mask_Conv2D(1, kernel_size=(3, 3),kernel_initializer=zeros_init,bias_initializer=zeros_init,
                         activation='relu')(input,inputs2=sim12,repeat=1,mode='normal')
    conv_knn2 = mask_Conv2D(1, kernel_size=(3, 3),kernel_initializer=zeros_init,bias_initializer=zeros_init,
                         activation='relu')(input,inputs2=sim_knn,repeat=1,mode='knn')
    with tf.Session() as s:
        s.run(tf.global_variables_initializer())
        o = s.run(conv_norm2)
        o_knn = s.run(conv_knn2)
    t1=o[:,:,:,0]
    t=o_knn[:,:,:,0]
    t2=i[:,:,:,0]
    
    # test mask_Conv3D
    sim = np.ones((2,3,3,3))
    sim12 = tf.constant(sim, dtype=tf.float32)
    sim_knn = np.ones((2,5,5,5), dtype=np.float32)
    sim_knn = tf.constant(sim_knn, dtype=tf.float32)
    i = np.round(np.random.random((2, 5, 5, 5, 2))*100)
    input = tf.constant(i, dtype=tf.float32)
    conv_norm3 = mask_Conv3D(1, kernel_size=(3, 3, 3),kernel_initializer=zeros_init,
                         activation='relu')(input,inputs2=sim12,repeat=1,mode='normal')
    conv_knn3 = mask_Conv3D(1, kernel_size=(3, 3, 3),kernel_initializer=zeros_init,bias_initializer=zeros_init,
                         activation='relu')(input,inputs2=sim_knn,repeat=1,mode='knn')    
    with tf.Session() as s:
        #print(sim_knn.eval())
        s.run(tf.global_variables_initializer())
        o = s.run(conv_norm3)
        o_knn = s.run(conv_knn3)    
    t1=o[0,:,:,:,0]
    t=o_knn[0,:,:,:,0]
    t2=i[0,:,:,:,0]
        
    
    sys.exit(0)
    
    
    
    sim = np.ones((3,3,3))
    sim12 = tf.constant(sim, dtype=tf.float32)
    #i = np.round(np.random.random((1, 1, 10, 11, 12))*100)
    i = np.round(np.random.random((1, 10, 11, 12, 1))*100)
    input = tf.constant(i, dtype=tf.float32)
    
    conv22 = mask_Conv3D(1, kernel_size=(3, 3, 3),activation='relu')(input,inputs2=sim12,repeat=1)
#    conv22 = mask_Conv3D(32, kernel_size=(3, 3, 3),activation='relu',input_shape=tensor2_shape)(input,sim12,2)
#    bias_init = tf.constant_initializer(0)
#    output = mask_Conv3D(input, 1, (3, 3, 3),
#        data_format='channels_first',
#        kernel_initializer=tf.constant_initializer(0),
#        bias_initializer=bias_init,
#        name='conv4d_valid')

#    a = tf.Variable([10, 20])
#    b = tf.assign(a, [20, 30])
#    c = a + [10, 20]
#    with tf.Session() as sess:
#        sess.run(tf.global_variables_initializer())
#        print("test_1 run a : ",sess.run(a)) # => [10 20]
        

    with tf.Session() as s:
        #print(sim12[2,2,2].eval())

        s.run(tf.global_variables_initializer())
        o = s.run(conv22)
        sys.exit(0)
        print("conv4d at (0, 0, 0, 0): %s"%o[0,0,0,0,0,0])
        i0 = i[0,0,0:2,0:2,0:2,0:2]
        print("manually computed value at (0, 0, 0, 0): %s"%i0.sum())

        k0 = tf.get_default_graph().get_tensor_by_name(
            'conv4d_valid_3dchan0/kernel:0').eval().flatten()
        k1 = tf.get_default_graph().get_tensor_by_name(
            'conv4d_valid_3dchan1/kernel:0').eval().flatten()
        k2 = tf.get_default_graph().get_tensor_by_name(
            'conv4d_valid_3dchan2/kernel:0').eval().flatten()

        print("conv4d at (0, 0, 0, 0): %s"%o[0,0,0,0,0,0])
        i0 = i[0,0,0,0:3,0:3,0:3].flatten()
        i1 = i[0,0,1,0:3,0:3,0:3].flatten()
        i2 = i[0,0,2,0:3,0:3,0:3].flatten()

        compare = (i0*k0 + i1*k1 + i2*k2).sum()
        print("manually computed value at (0, 0, 0, 0): %s"%compare)

        print("conv4d at (4, 4, 4, 4): %s"%o[0,0,4,4,4,4])
        i0 = i[0,0,4,4:7,4:7,4:7].flatten()
        i1 = i[0,0,5,4:7,4:7,4:7].flatten()
        i2 = i[0,0,6,4:7,4:7,4:7].flatten()

        compare = (i0*k0 + i1*k1 + i2*k2).sum()
        print("manually computed value at (4, 4, 4, 4): %s"%compare)


    i = np.round(np.random.random((1, 1, 10, 11))*100)
    input = tf.constant(i, dtype=tf.float32)
    bias_init = tf.constant_initializer(0)
    conv21 = mask_Conv2D(32, kernel_size=(3, 3),activation='relu',
                 input_shape=tensor1_shape)(input11,dense21,2)
    output = mask_Conv2D(
        input,
        2,
        (3, 3),
        data_format='channels_first',
        padding='same',
        kernel_initializer=tf.constant_initializer(1),
        bias_initializer=bias_init,
        name='conv4d_same')

    with tf.Session() as s:

        s.run(tf.global_variables_initializer())
        o = s.run(output)
        
        k10 = tf.get_default_graph().get_tensor_by_name(
            'conv4d_same_3dchan0/kernel:0').eval().flatten()
        k11 = tf.get_default_graph().get_tensor_by_name(
            'conv4d_same_3dchan1/kernel:0').eval().flatten()
        k12 = tf.get_default_graph().get_tensor_by_name(
            'conv4d_same_3dchan2/kernel:0').eval().flatten()

        print("conv4d at (0, 0, 0, 0): %s"%o[0,0,0,0,0,0])
        i0 = i[0,0,0:2,0:2,0:2,0:2]
        print("manually computed value at (0, 0, 0, 0): %s"%i0.sum())

        print("conv4d at (5, 5, 5, 5): %s"%o[0,0,5,5,5,5])
        i5 = i[0,0,4:7,4:7,4:7,4:7]
        print("manually computed value at (5, 5, 5, 5): %s"%i5.sum())

        print("conv4d at (9, 10, 11, 12): %s"%o[0,0,9,10,11,12])
        i9 = i[0,0,8:,9:,10:,11:]
        print("manually computed value at (9, 10, 11, 12): %s"%i9.sum())